package com.javajun.spider.bt7086.processor;

import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import javax.annotation.PostConstruct;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.javajun.spider.bt7086.common.UrlUtils;
import com.javajun.spider.bt7086.entity.Video;
import com.javajun.spider.bt7086.pipeline.VideoPipeline;
import com.javajun.spider.bt7086.repository.VideoRepository;
import com.javajun.spider.bt7086.util.UserAgent;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

@Component
public class Bt7086PageProcessor implements PageProcessor {
	
	@Autowired
	private VideoRepository videoRepository;
	
	private static Bt7086PageProcessor pageProcessor;
	
	@PostConstruct 
	public void init() {
		pageProcessor = this;
		pageProcessor.videoRepository = this.videoRepository;
	}

	private Site site = Site.me().setTimeOut(20 * 1000) // 超时时间，毫秒
			.setRetryTimes(3) // 失败重试次数
			.setCharset("UTF-8")
			.setUserAgent(UserAgent.Chrome.getAgent());

	// http://k6.csnmdcjnx.com/pw/thread.php?fid=111
	// http://k6.csnmdcjnx.com/pw/thread.php?fid=111&page=2
	private static final String URL_LIST = "\\w*thread\\.php[?]fid=\\d*(?:&page=\\d*|)";

	// http://k6.csnmdcjnx.com/pw/html_data/111/2001/4545631.html
	private static final String URL_DETAIL = "\\w*html_data/\\d*/\\d*/\\d*\\.html";

	@Override
	public void process(Page page) {

		Selectable pageUrl = page.getUrl();
		Html pageHtml = page.getHtml();

		if (pageUrl.regex(URL_LIST).match()) {
			
			// 这里面获取到的链接实际上都是会重复，即一个链接会出现两次。
			// 但是在这里面不需要去重，因为加入爬取列表的时候，会进行相应的去重处理
			
			List<String> urlDetailListRepeat = pageHtml.xpath("//div[@class='t z']").links().regex(URL_DETAIL).all();
			List<String> urlDetailList = urlDetailListRepeat.stream().distinct().collect(Collectors.toList());
			Iterator<String> urlDetailListIterator = urlDetailList.iterator();
			while(urlDetailListIterator.hasNext()) {
				String url = urlDetailListIterator.next();
				List<Video> videoList = pageProcessor.videoRepository.findByUrl(url);
				if(!videoList.isEmpty()) {
					urlDetailListIterator.remove();
				}
			}
			
			 page.addTargetRequests(urlDetailList);

			// 下一页的链接，因为底层是Jsoup，所以webmagic的xpath不支持兄弟节点的选择，只能使用css选择器
			// class 为 pages cc 的 div 下的 b 标签后面的一个 a 标签，且这个 a 标签的类（class）不为b
			List<String> urlListRepeat = pageHtml.css("div.pages.cc > b + a:not(.b)").links().regex(URL_LIST).all();
			List<String> urlList = urlListRepeat.stream().distinct().collect(Collectors.toList());
			page.addTargetRequests(urlList);
			
			// 无需业务处理，不需要进入Pipeline进行保存数据
			page.setSkip(true);
			
		}
		
		if (pageUrl.regex(URL_DETAIL).match()) {
			Video model = new Video();

			// http://k6.csnmdcjnx.com/pw/html_data/111/2001/4545631.html
			// page.putField("url", pageUrl.get());
			// model.setUrl(pageUrl.get());
			
			// /html_data/111/2001/4545631.html
			String urlStr = UrlUtils.getPath(page.getUrl().get());
			model.setUrl(urlStr);

			String title = pageHtml.xpath("//span[@id='subject_tpc']/text()").get().trim();
			// page.putField("title", title);
			model.setTitle(title);

			// 发表于: 2020-01-11 22:54
			String publicTime = pageHtml.xpath("//div[@class='tiptop']//span[@class='fl gray']/text()").get().trim()
					.substring(5);
			// page.putField("publicTime", publicTime);
			model.setPublicTime(publicTime);

			// https://baiduyunbo.com/?id=ID5N3zsW
			String iframeLink = pageHtml.xpath("//div[@id='read_tpc']/iframe/@src").get();
			// page.putField("iframeLink", iframeLink);
			model.setIframeLink(iframeLink);

			String m3u8Id = Pattern.compile("^http(s)?://\\w*\\.\\w*/[?]id=").matcher(iframeLink).replaceAll("");
			//  page.putField("m3u8Id", m3u8Id);
			model.setM3u8Id(m3u8Id);
			page.putField("model", model);
			// 需要保存数据，进入pipeline
			 page.setSkip(false);
		}
		
	}

	@Override
	public Site getSite() {
		return site;
	}
	
	//http://x1.pbnmdssb.xyz/pw/thread.php?fid=111&page=385
	public void run() {
		Spider.create(new Bt7086PageProcessor())
		.addUrl("http://k6.csnmdcjnx.com/pw/thread.php?fid=111")
		.addPipeline(new ConsolePipeline()).addPipeline(new VideoPipeline())
		.setScheduler(new FileCacheQueueScheduler("webmagic-urls"))
		.thread(3)
		.run();
		
	}

}
